import urllib
import struct
import os
import os.path
import sys
import string
import time
from HTMLParser import HTMLParser
import json
import shutil
import datetime
from datetime import timedelta
from utils import *
import threading
import MySQLdb
import ConfigParser

class AppConfig:
    def __init__(self):
        self.config = {}

    def load_config(self):
        config = ConfigParser.RawConfigParser()
        config.read('config.ini')
        return config

class DailyFetcher:
	def __init__(self, config, execution_date, web_fetcher):
		self.config = config
		self.sql_path = config.get('path', 'sql_path')
		self.execution_date = execution_date
		self.web_fetcher = web_fetcher

	def run(self):
		utils = MyUtils();
		url = self.config.get('url', 'fund_url_daily') 	
		url = url +'date=' + self.execution_date
		i = 1 # types of all funds
		result = []
		while i <= 1:
			url = url+'&type=' + str(i)
			i = i + 1
			html = self.web_fetcher.get_web_data(url)		
			html = json.loads(html)
			items = html['data']
			m = 0
			for item in items:
				m = m+1
				result.append(item['fundCode'])
		return result

class BaseInfoFetcher:

	def __init__(self, config, sql_file, web_fetcher, adapter):
		self.config = config
		self.web_fetcher = web_fetcher
		self.adapter = adapter
		self.sql_file = sql_file

	def run(self, items):
		for fund_code in items:
			url = config.get('url', 'fund_url_base');
			url = url+'fundcode=' + fund_code
			print url
			html = self.web_fetcher.get_web_data(url)
			path = config.get('path','init_path')
			file_name = path + 'base_info_' + fund_code
			file = open(file_name, 'a')
			file.write(html)
			file.close()
			parser = FundHTMLParser()
			all_data = parser.read(html);
			data = self.get_base_info(all_data) 
			self.adapter.save_base_info(data, self.sql_file)
	def get_base_info(self, data):
		result = []
		i = 0 
		while i<16:
			if (i % 2 == 1):
				result.append(data[i])
			i = i+1
		return result

class HistoryFetcher:

	def __init__(self, config, is_initial, sql_file, web_fetcher, adapter):
		self.config = config
		self.is_initial = is_initial
		self.web_fetcher = web_fetcher
		self.adapter = adapter
		self.sql_file = sql_file

	def run(self, items):
		for fund_code in items:
			url = config.get('url', 'fund_url_history');
			url = url+'fundcode=' + fund_code
			print url
			html = self.web_fetcher.get_web_data(url)
			path = config.get('path','init_path')
			parser = FundHTMLParser()
			html = parser.read(html);
			html = self.get_base_info(html) 
			i = 0
			sql_value = []
			for item in html:
				if i % 8 ==0:
					if sql_value != []:
						self.adapter.save_history_value(sql_value, self.sql_file)
					sql_value = []
				sql_value.append(item)
				i = i+1

	def get_base_info(self, data):
		result = []
		if self.is_initial:
			i = 0
			for item in data:
				if i>=7:
					result.append(item)
				i = i +1
		else:
			m = 7;
			while m<=15 :
				m = m + 1
				result.append(data[m])

		return result

	def get_not_exist_funds(self, fund_items):
		sql = 'select fund_code from fud_base'
		db_records = self.adapter.get_records(sql)	
		result = []
		for item in fund_items:
			exist = false
			for record in db_records:
				if item == record:
					exist = true
					break
			if not exist:
				result.append(item) 
		return result

class FundInfo:
	def __init__(self, config, is_inital):
		self.config = config
		self.sql_path = config.get('path', 'sql_path')
		self.is_initial = is_initial
		self.thread_count = int(config.get('threads', 'thread_count'))

	def run(self):
		utils = MyUtils()
		execution_date =  utils.getYesterday()

		web_fetcher = WebFetcher()
		daily = DailyFetcher(self.config, execution_date, web_fetcher)
		fund_items = daily.run()

		divided_items = utils.divide_array(fund_items, int(self.thread_count))
		thread_pool = []
		index = 0 
		while index < self.thread_count-1:
			th = threading.Thread(target=self.work, args=(self.config, self.is_initial,divided_items[index], index, execution_date))
			thread_pool.append(th)
			index = index + 1		

		for t in thread_pool:
			t.start()


	def work(self, config, is_initial,  fund_items, thread_index, execution_date):
		sql_base_info_file = self.sql_path + 'sql_base_info_'+execution_date+'_'+str(thread_index)
		sql_history_value_file = self.sql_path + 'sql_history_value_'+execution_date+'_'+str(thread_index)
		
		f_sql_base_info = open(sql_base_info_file, 'a')
		f_sql_history_value = open(sql_history_value_file,'a') 

		web_fetcher = WebFetcher()
		adapter = MySQLAdapter(config)
		print 'thread_index:' + str(thread_index)
		print fund_items
			
		if self.is_initial:
			baseInfoFetcher = BaseInfoFetcher(config, f_sql_base_info, web_fetcher, adapter)
			historyFetcher = HistoryFetcher(config, is_initial, f_sql_history_value, web_fetcher, adapter)
			baseInfoFetcher.run(fund_items)
			historyFetcher.run(fund_items)
		else:
			historyFetcher = HistoryFetcher(config, is_initial, f_sql_history_value, web_fetcher, adapter)
			not_exist_fund_items = historyFetcher.get_not_exist_funds(fund_items)
			if not_exist_funds != []:
				baseInfoFetcher = BaseInfoFetcher(config, f_sql_base_info, web_fetcher, adapter)
				baseInfoFetcher.run(not_exist_funds)
			historyFetcher.run(fund_items)
				
		f_sql_base_info.close()
		f_sql_history_value.close()

# main entry.
if __name__ == "__main__":

		config = AppConfig()
		config = config.load_config()
		is_initial = config.get('initial', 'is_initial')
		fetcher = FundInfo(config, is_initial);
		fetcher.run()	

